package com.terren.spider.core.html.processor;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import com.terren.spider.core.html.configuration.Configuration;
import com.terren.spider.core.html.scheme.ICustomScheme;
import com.terren.spider.entity.config.SysConfig;
import com.terren.spider.util.common.BeanUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 列表页 processor
 * @author chenyu
 *
 */
public class HtmlUrlProcessor extends BasicHtmlProcessor implements PageProcessor {

	public HtmlUrlProcessor(Configuration configuration) throws Exception {
		super(configuration);
	}

	@Override
	public void process(Page page) {
		logMessage(page);
		/***
		 * 抓取目标页链接
		 */
		if (!this.isTargetUrl(page, configuration.getEntry())) {// 如果不是目标页面
			Integer recordLimit = configuration.getEntry().getRecordLimit();
			Integer recordSize = configuration.getPageRecordSize() != null ? configuration.getPageRecordSize() : SysConfig.RECORD_SIZE;
			List<String> requestUrl = new ArrayList<>();
			if (isContainHelpUrl(configuration.getEntry())) {// 如果是入口页 ，如果有列表页
				requestUrl.addAll(this.findHelpUrls(page, configuration.getEntry()));// 添加列表页

				/**
				 * 自定义查找helpurl方案(如分页等)
				 */
				for (ICustomScheme scheme : configuration.getSchemes()) {// 自定义列表页方案。等
					if (BeanUtil.includeMethod("findHelpUrl", scheme.getClass())) {
						Set<String> urls = scheme.findHelpUrl(page, configuration.getEntry());
						log.info("[entry:"+configuration.getEntry().getEntryId()+"]"+"[scheme:"+scheme.getClass().getName());
						log.info("[method:findHelpUrl]"+"[size:"+urls.size()+"]");
						if (null != urls) {
							requestUrl.addAll(urls);
						}
					}
				}
			}
			
			if ((null!=recordLimit)) {// 分页数量
				if ((getHelpPagingUrlCount() < ((recordLimit + SysConfig.MORE_SIZE) / recordSize - 1))) {
					requestUrl.addAll(this.findHelpPaginationUrl(page,  configuration));// 分页
				}
			} else {
				requestUrl.addAll(this.findHelpPaginationUrl(page,configuration));//分页
			}
			helpUrls.addAll(requestUrl);
			super.addTargetRequests(page,helpUrls);
		}
		this.findTargetUrls(page, configuration.getEntry());//寻找目标页 放到set中
		/***
		 * 如果入口页就是目标页
		 */
		if (this.isTargetUrl(page, configuration.getEntry())) {// 如果页面是 目标页
			try {
				targetUrls.add(getCurrentPageUrl(page));//加入到targeturl队列中
			} catch (Exception e) {
				log.error("process parse error", e);
			}
		}
	}
}
